﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace HomeWork9_WinForm
{
    public delegate void DownLoadEventHandler(string url);
    public class Crawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        public DownLoadEventHandler PageDownloaded;
        public string startUrl = "http://www.cnblogs.com/dstang2000/";

        /*static void Main(string[] args) {
      Crawler myCrawler = new Crawler();
      
      if (args.Length >= 1) startUrl = args[0];
      myCrawler.urls.Add(startUrl, false);//加入初始页面
      new Thread(myCrawler.Start).Start();
    }*/

        public void Start()
        {
            PageDownloaded("开始爬行了.... ");
            urls[startUrl] = false;
            while (true)
            {               
                string current = null;                
                foreach (string url in urls.Keys)
                {                    
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 15) break;
                PageDownloaded("爬行" + current + "页面!");
                string html = Download(current); // 下载
               
                urls[current] = true;
                count++;
                Parse(html, current);//解析,并加入新的链接
                PageDownloaded("爬行结束");
            }
        }

        public string Download(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);

                return html;
            }
            catch (Exception ex)
            {
                PageDownloaded(ex.Message);
                return "";
            }
        }

        private void Parse(string html, string currentUrl)
        {
            //获取当前网址
            MatchCollection currentUrlSlash = new Regex("/").Matches(currentUrl);
            int startUrlSlashIndex3 = currentUrlSlash[2].Index;
            string strRef = @"(href|HREF)[ ]*=[ ]*[""'][^""'#>]+(html|htm|htm?)[""']";
            //string strRef = @"(href|HREF)[ ]*=[ ]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                          .Trim('"', '\"', '#', '>');
                if (strRef.Length == 0) continue;
                //判断是否为绝对地址
                if (!new Regex(@"^(\w|\s)+://").IsMatch(strRef))
                {
                    //href=“blog/paper1.html”
                    if (new Regex(@"^\w+").IsMatch(strRef))
                    {
                        strRef = currentUrl + strRef;
                    }
                    //href =“/ blog / paper1.html” 
                    //href=“..og/zhang/”
                    else
                    {
                        MatchCollection strRefSlash = new Regex("/").Matches(strRef);

                        int strRefSlashIndex = 0;
                        if (strRefSlash.Count != 0) strRefSlashIndex = strRefSlash[0].Index;
                        strRef = currentUrl.Substring(0, startUrlSlashIndex3)
                         + strRef.Substring(strRefSlashIndex);

                    }
                    if (urls[strRef] == null) urls[strRef] = false;
                }
                //绝对地址
                else
                {

                    if (strRef.Contains(currentUrl.Substring(0, startUrlSlashIndex3)) ||
                        strRef.Contains(currentUrl.Substring(0, startUrlSlashIndex3).
                                   Replace("http", "https")))
                        if (urls[strRef] == null) urls[strRef] = false;
                }




            }
        }
    }
}
